package com.spider.ask120.webmagic;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.jdiy.core.Rs;
import org.jdiy.json.JsonArray;
import org.jdiy.json.JsonObject;
import org.jdiy.json.JsonUtil;

import com.spider.ask120.service.ArticleService;
import com.spider.ask120.service.ChannelService;
import com.spider.utils.HttpClientUtil;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

/**
 * 健康经验正文
 * @author Jxw
 *
 */
public class Jingyan  implements PageProcessor{
	static String jkjyUrl = "http://m.120ask.com/jingyan/";
	static String listUrl = "http://m.120ask.com/jingyan/list/\\w+"; 

	  
	public static void main(String[] args) {
		List<Rs> channels = ChannelService.getChannelLowCatalog();
		for(Rs channel : channels){
			String id = channel.get("id");
			String str_id = channel.get("str_id");
			Spider.create(new Jingyan()).addUrl("http://m.120ask.com/jingyan/list/"+str_id+"?cata_id="+id).thread(1).run(); 
		}
//		Spider.create(new Jingyan()).addUrl("http://m.120ask.com/jingyan/list/1kwvmhsksk?cata_id=1928").thread(1).run();
    }
	
 
	public void process(Page page) { 
		if(page.getUrl().regex(listUrl).match()) {//不包括.html后缀  //+"$"  
			//获取?cata_id= 后面的栏目ID
			String path = page.getUrl().toString();
			String cata_id = path.substring(path.indexOf("?cata_id=")+9);
			
			
			//无下级栏目，开始加载分页数据列表，获取正文URL
			Map<String, String> params = new HashMap<String, String>();
			params.put("ajax", "1");
			int pageIndex = 1;	
			while(true){ 
				//从第一页开始取， 只能拿到HTML页面内容，取不到json格式的数据，所以跳过第一页；
				pageIndex ++;
				params.put("page", pageIndex+"");
				String listJson = HttpClientUtil.getInstance().sendHttpPost(page.getUrl().toString(), params);
				if(listJson != null && !listJson.isEmpty()){  
					try {
						JsonObject rstObj = (JsonObject) JsonUtil.parse(listJson); //{"code":200,"list":[]}
						if(rstObj != null && rstObj.get("code") != null && "200".equals(rstObj.get("code").toString()) 
										&& rstObj.get("list") != null && !"[]".equals(rstObj.get("list").toString())){
							
							JsonArray objAry = (JsonArray) JsonUtil.parse(rstObj.get("list").toString());
							if(objAry.size() > 0){ 
								for(int i = 0; i < objAry.size(); i++){
									JsonObject obj = (JsonObject) objAry.get(i); 
									Rs article = new Rs("article");  
									String str_id = obj.get("id_str").toString();
									str_id = str_id.substring(0, str_id.indexOf("."));
									article.set("str_id", str_id);
									article.set("channel_id", cata_id);
									article.set("title", obj.get("title").toString());
									article.set("abst", obj.get("content").toString());
									article.set("pic", obj.get("pic").toString());
									ArticleService.saveArticle(article);
									
									//进入到文章正文
									Map<String, Object> extras = new HashMap<String, Object>();
									extras.put("article_id", article.get("id")); 
									Request request = new Request("http://m.120ask.com/jingyan/"+obj.get("id_str").toString()+".html");
									request.setExtras(extras);
									page.addTargetRequest(request);
								}
							}else{ 
								break;
							}
						}else{ 
							break;
						}
					} catch (Exception e) { 
						e.printStackTrace();
					}
				}
				
				if (pageIndex == 6) {
					//只取一页的数据
					break;
				}
			}
			
			 
		}else if(page.getUrl().regex(jkjyUrl+"\\w+.html").match()) {
			String title = page.getHtml().xpath("//div[@class='g-jinyan-topic']/h3/text()").get();
			String create_time = page.getHtml().xpath("//div[@class='g-jinyan-topic']/p[1]/text()").regex("\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}").get();
			String share = page.getHtml().xpath("//div[@class='g-jinyan-topic']/p[1]/text()").toString();
			share = share.substring(share.indexOf("由")+1, share.indexOf("分享")).trim(); 
			String abst = page.getHtml().xpath("//div[@class='g-jinyan-cont']/p/text()").get();			//介绍
			String remind = page.getHtml().xpath("//div[@class='g-jinyan-conf']/p[2]/html()").get();	//注意事项
			List<String> contStep = page.getHtml().xpath("//div[@class='d1 img']/dl/dd/html()").all();		//步骤方法
			List<String> keywords = page.getHtml().xpath("//div[@class='g-jinyan-topic']/p[2]/a/text()").all();
			

			String article_id = page.getRequest().getExtra("article_id").toString();
			Rs article = ArticleService.getArticleById(article_id);
			if(article != null && !article.isNull()){ 
				article.set("title", title);
				article.set("create_time", create_time);
				article.set("source", share); 
				
				if(keywords.size() > 0){ 
					String keyword = "";
					for(String kw : keywords){
						keyword += keyword.isEmpty() ? kw : ","+kw;
					}
					article.set("keyword", keyword); 
				} 
				JsonArray stepArr = new JsonArray();
				for(int i = 0; i < contStep.size(); i++){
					JsonObject step = new JsonObject();
					step.set("step_index", (i+1));
					step.set("step_content", contStep.get(i)); 
					stepArr.add(step);
				}
				
				JsonObject content = new JsonObject();
				content.set("abst", abst);
				content.set("remind", remind); 
				content.set("steps", stepArr);
				
				article.set("content", content.toString());
				ArticleService.updateArticle(article);
			}
			  
 
		} 
	}

	// 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
	private Site site = Site.me().setRetryTimes(3).setSleepTime(1);
	public Site getSite() {  
		return site;
	}
}
